###############################################################################

# Replication File (Experiment): "The Effect of Simultaneous Proposals"
# Klaudia Wegschaider, February 2025

###############################################################################

# STRUCTURE

# This file covers the replication of the experimental data.
# For the replication of the observational data, please see the additional file.

# In the first part, I will cover all the analyses that are included in
# the main text of the paper.

# In the second part, I will cover all the analyses that are included in the
# appendices.

# I do not explain the purpose of all the steps here. Readers should consult
# this file in combination with the manuscript and appendix.

###############################################################################

# BACKGROUND

# This code is based on R version 4.1.3 (2022-03-10)

# Please load the required packages.

# Packages
library(purrr) 
library(plyr) 
library(tidyverse) 
library(ggplot2)
library(haven)
library(broom) 
library(pander)
library(plm)
library(knitr)
library(kableExtra)
library(xtable)

# sessionInfo()
# R version 4.1.3 (2022-03-10)
# [1] xtable_1.8-4     kableExtra_1.4.0 knitr_1.47       plm_2.6-1        pander_0.6.5    
# [6] broom_0.7.12     haven_2.4.3      forcats_0.5.1    stringr_1.5.0    dplyr_1.1.2     
# [11] readr_2.1.2      tidyr_1.3.0      tibble_3.2.1     ggplot2_3.5.1    tidyverse_1.3.2 
# [16] plyr_1.8.7       purrr_1.0.1 

###############################################################################

# DATA

# Please load the required data
exp_data <- read.csv(file="experiment_data.csv") 
View(exp_data)

# This data was collected in February 2022. 
# Respondents were recruited via lucid.

# Pre-treatment variables:
# Respondent: Individual number per respondent, assigned afterwards (1-2000)
# StartDate: Survey starting time
# EndDate: Survey ending time
# duration: duration in seconds
# Age: 18-24, 25-34, 35-44, 45-54, 55-64, 65+
# Identity: female, male, nonbinary, prefer not
# State: list US states

# Experimental part:
# Respondents were were either in the treatment group or in the control group (C).
# The treatment group was further sub-divided into A and B (proposal order).
# A) extreme_treat1: Proposition 1: Should immigrants who have been legally residing in your state for at least one year have the right to vote in local elections?
# A) moderate_treat1: Proposition 2: Should immigrants who have been legally residing in your state for at least eight years have the right to vote in local elections?
# A) Q4c: Tie-breaker question: In case both proposals are supported by more than 50% of voters, which of the two proposals would you prefer to be implemented?
# B) moderate_treat2: Proposition 1: Should immigrants who have been legally residing in your state for at least eight years have the right to vote in local elections?
# B) extreme_treat2: Proposition 2: Should immigrants who have been legally residing in your state for at least one year have the right to vote in local elections?
# B) Q5c: Tie-breaker question: In case both proposals are supported by more than 50% of voters, which of the two proposals would you prefer to be implemented?
# C) moderate_control: Proposition 1: Should immigrants who have been legally residing in your state for at least eight years have the right to vote in local elections?

# Post-treatment variables:
# Type: Which right for immigrants did you just vote on?
# Education: What is the highest level of school you have completed or the highest degree you have received?
# Party: Generally speaking, do you think of yourself as a Republican, a Democrat, an Independent, or something else?
# MigAttitude: Is the United States made a worse or a better place to live by people coming to live here from other countries? 
# Ethnicity variables: (self-explanatory)

# Additionally coded variables:
# treated: dummy variable for whether or not in the treatment group
# moderate_treated: support for moderate proposal in the treatment group
# moderate_support: support for moderate proposal (control and treatment group)
# extreme_treated: support for extreme proposal in the treatment group
# MigAttitude_simple: simplified to three options (better, neutral, worse)
# Region (self-explanatory, coded based on state)
# Partisanship: simplified to three options (Democrat, Republican, Unaligned/Independent)
# attention: failed or passed attention check (see Type)
# forbalance: dummy created for balance test purpose
# extreme_pref: of those in the treatment group, who preferred the liberal proposal?

###############################################################################

# PART ONE: ANALYSES/RESULTS INCLUDED IN THE MAIN TEXT

# PARTISAN PROPORTIONS (INFO IN FLOW TEXT)
table(exp_data$Partisanship)
## Democrat: 890
## Republican: 668
## Independent: 442

# What is the share of Democrats in the sample?
(890/2000)*100 
## 44.5

# What is the share of Republicans in the sample?
(668/2000)*100
## 33.4

# What is the share of Independents in the sample?
(442/2000)*100
## 22.1

###

# MAIN RESULT (TABLE 2)

# one-tailed hypothesis test
t1 <- t.test(exp_data$moderate_treated, exp_data$moderate_control, 
             alternative = "greater")
t1
# 4.2 percentage points; p-value = 0.01678

# two-tailed hypothesis test
t2 <- t.test(exp_data$moderate_treated, exp_data$moderate_control)
t2
# 4.2 percentage points; p-value = 0.03356

# table showing results with both t1 and t2
tab <- map_df(list(t1, t2), tidy)
tab <- tab[c("alternative", "estimate", "statistic", "p.value", "conf.low",
             "conf.high")]

# modifying the table for better visual presentation
tab <- tab %>% # rounding to three decimal places
  mutate_if(is.numeric, ~ round(., 3))
tab$alternative <- gsub("\\.", "-", tab$alternative) # replacing the period in "two.sided" with a hyphen

# Table 2
tab

###

# POTENTIAL COUNTER-EFFECT AMONG A SUBSET (INFO IN FLOW TEXT)

# How many people actually preferred the extreme proposal among those who saw both?
# Extreme version preference (coded based on tiebreaker question)
mean(exp_data$extreme_pref, na.rm=TRUE)
# Almost 30% of people in the treatment groups preferred the extreme option.

# What is the support for the moderate proposal among people who prefer the extreme option?
mod_among_extreme <- filter(exp_data, extreme_pref==1)
mean(mod_among_extreme$moderate_treated) 
# among those who preferred the extreme proposal in the tie-breaker question,
# the support for the moderate proposal was: 0.8061224
mean(mod_among_extreme$extreme_treated) 
# among those who preferred the extreme proposal in the tie-breaker question,
# the support for the extreme proposal was: 0.8979592

# Insight: There were voters who rejected the moderate proposal, but
# preferred and endorsed the extreme proposal.

# So, yes, there appears to have been a counter-balancing effect.

# Does this liberal preference correspond to partisanship?
table(exp_data$extreme_pref, exp_data$Partisanship)
# Democrats most likely to prefer liberal option. (172; 274)
# Republicans least likely to prefer liberal option. (59; 269)

# If there was a counter-effect among Democrats, this could partly explain 
# why the treatment was only effective among Republicans.

###

# HETEROGENEOUS TREATMENT EFFECTS BY PARTISANSHIP (BACKGROUND FIGURE 2)

## Partisanship

# Without covariates:
# Y = \alpha + \beta_1(Treatment) + \beta_2(Partisanship) + \beta_3(Treatment)(Partisanship) + \epsilon
m_parties <- lm(moderate_support ~ treated + Partisanship + treated * Partisanship, data=exp_data)
summary(m_parties)
# Negative effect (stat. sig. at 0.001 level) of being Republican
# BUT: Positive effect (stat. sig. at 0.01 level) of being Republican and in the treatment group
# This suggests a heterogeneous treatment effect by partisanship.

# With covariates:
# Y = \alpha + \beta_1(Treatment) + \beta_2(Partisanship) + \beta_3(Treatment)(Partisanship) + [Covariates] + \epsilon
m_parties_cov <- lm(moderate_support ~ treated + Partisanship + treated * Partisanship + Gender + Agecohort + State + White + Black + AmIndian_AlNative + Asian + NatHawaiian_Pacific + Hispanic_Latinx + MiddleEastern + Other + Prefer_not + Education, data=exp_data)
summary(m_parties_cov)
# Adding covariates further enhances the heterogeneous treatment effect by partisanship.
# This is because of partisan fluctuation across states (which are not perfectly balanced)

# Subsetting by partisanship
Republicans <- filter(exp_data, Partisanship=="Republican")
Unaligned <- filter(exp_data, Partisanship=="Independent/Unaligned")
Democrats <- filter(exp_data, Partisanship=="Democrat")

m_basic <- lm(exp_data$moderate_support ~ exp_data$treated)
summary(m_basic)

m_basic_rep <- lm(Republicans$moderate_support ~ Republicans$treated)
summary(m_basic_rep)

m_basic_dem <- lm(Democrats$moderate_support ~ Democrats$treated)
summary(m_basic_dem)

m_basic_ind <- lm(Unaligned$moderate_support ~ Unaligned$treated)
summary(m_basic_ind)

# Zooming in on Republicans
# What is the support for the moderate proposal in Republicans (by treatment status)?
class(exp_data$treated)
treatment <- filter(exp_data, treated==1)
control <- filter(exp_data, treated==0)
Rep_treatment <- filter(treatment, Partisanship=="Republican")
Rep_control <- filter(control, Partisanship=="Republican")

mean(Rep_control$moderate_support) # 55%
mean(Rep_treatment$moderate_support) # 68%

# Difference between treatment-group and control-group Republicans is 
# statistically significant (p-value = 0.000541)
t.test(Rep_control$moderate_support, Rep_treatment$moderate_support)

# Zooming in on Democrats
# What is the support for the moderate proposal Democrats (by treatment status)?
Dem_treatment <- filter(treatment, Partisanship=="Democrat")
Dem_control <- filter(control, Partisanship=="Democrat")

mean(Dem_control$moderate_support) # 83.1% (strong Democrats: 82.7%)
mean(Dem_treatment$moderate_support) # 82.7% (strong Democrats: 83.5%)

# Difference between treatment-group and control-group Democrats is
# NOT statistically significant (p-value = 0.8827)
t.test(Dem_treatment$moderate_treated, Dem_control$moderate_control)

###

# HETEROGENEOUS TREATMENT EFFECTS BY MIGRATION ATTITUDE (BACKGROUND FIGURE 3)

## Migration Attitude

# Is the US made a worse or better place to live by people coming to live here
# from other countries? (better=1, neutral=2, worse=3)
table(exp_data$MigAttitude_simple)
exp_data$MigAttitude_simple <- as.factor(exp_data$MigAttitude_simple)

# Without covariates
# Y = \alpha + \beta_1(Treatment) + \beta_2(MigAttitude) + \beta_3(Treatment)(MigAttitude) + \epsilon
m_migatt_sim <- lm(moderate_support ~ treated + MigAttitude_simple + treated * MigAttitude_simple, data=exp_data)
summary(m_migatt_sim)
# anti-immigration people are against enfranchisement
# but: positive effect (stat. sig. at 0.05 level) on anti-immigrant people in the treatment group

# With covariates
# Y = \alpha + \beta_1(Treatment) + \beta_2(MigAttitude) + \beta_3(Treatment)(MigAttitude) + [Covariates] + \epsilon
m_migatt_sim_cov <- lm(moderate_support ~ treated + MigAttitude_simple + treated * MigAttitude_simple + Gender + Agecohort + State + White + Black + AmIndian_AlNative + Asian + NatHawaiian_Pacific + Hispanic_Latinx + MiddleEastern + Other + Prefer_not + Education, data=exp_data)
summary(m_migatt_sim_cov)
# anti-immigration people are against enfranchisement
# but: positive effect (stat. sig. at 0.05 level) on anti-immigrant people in the treatment group

# Zooming in on worse immigration people
# What is the support for the moderate proposal in worse immigration people (by treatment status)?
treatment <- filter(exp_data, treated==1)
control <- filter(exp_data, treated==0)
worse_treatment <- filter(treatment, MigAttitude_simple=="worse")
worse_control <- filter(control, MigAttitude_simple=="worse")

mean(worse_control$moderate_support, na.rm=TRUE)
mean(worse_treatment$moderate_support, na.rm=TRUE)

# Difference is statistically significant (p-value = 0.007062)
t.test(worse_treatment$moderate_treated, worse_control$moderate_control)

# Zooming in on better immigration people
# What is the support for the moderate proposal in better immigration people (by treatment status)?
better_treatment <- filter(treatment, MigAttitude_simple=="better")
better_control <- filter(control, MigAttitude_simple=="better")

mean(better_control$moderate_control, na.rm=TRUE)
mean(better_treatment$moderate_treated, na.rm=TRUE)

# Difference is not statistically significant
t.test(better_treatment$moderate_treated, better_control$moderate_control)

###############################################################################

# PART TWO: ANALYSES/RESULTS INCLUDED IN THE APPENDIX

# POWER CALCULATION

power.prop.test(
  n = NULL,
  p1 = 0.25,
  p2 = 0.3,
  sig.level = 0.05,
  power = 0.8,
  alternative = "one.sided")

###

# BALANCE TESTS

# class(exp_data$treated)
# data$forbalance <- as.numeric(exp_data$treated)

# table showing balance of pre-treatment covariates
balance_pre <- lm(exp_data$forbalance ~ exp_data$Gender + exp_data$Agecohort + exp_data$Region)
summary(balance_pre)

# table showing balance of post-treatment covariates
balance_post <- lm(exp_data$forbalance ~ exp_data$Education + exp_data$White + exp_data$Black + exp_data$AmIndian_AlNative + exp_data$Asian + exp_data$NatHawaiian_Pacific + exp_data$Hispanic_Latinx + exp_data$MiddleEastern + exp_data$Other + exp_data$Prefer_not)
summary(balance_post)

# table showing balance of pre- and post-treatment covariates combined
balance_combined <- lm(exp_data$forbalance ~ exp_data$Gender + exp_data$Agecohort + exp_data$Region + exp_data$Education + exp_data$White + exp_data$Black + exp_data$AmIndian_AlNative + exp_data$Asian + exp_data$NatHawaiian_Pacific + exp_data$Hispanic_Latinx + exp_data$MiddleEastern + exp_data$Other + exp_data$Prefer_not)
summary(balance_combined)

# State (zoomed in separately because too large for joint table)
class(exp_data$State)
balance_state <- lm(exp_data$forbalance ~ exp_data$State)
summary(balance_state) 
# balanced, except 15,16,21,23,26,31,34,37,4,42,44,45,48,5,50,51,6

###

# PRE-REGISTERED: ADDITIONAL ANALYSES (OLS)

# Basic OLS model
m_basic <- lm(moderate_support ~ treated, data = exp_data)
summary(m_basic)

# OLS model + attention check (interacted with treatment)
m_attention <- lm(moderate_support ~ treated * attention, data = exp_data)
summary(m_attention)

# OLS model + partisanship (interacted with treatment)
m_parties <- lm(moderate_support ~ treated * Partisanship, data = exp_data)
summary(m_parties)

# OLS model + migration attitude (interacted with treatment)
m_migatt_sim <- lm(moderate_support ~ treated * MigAttitude_simple, data = exp_data)
summary(m_migatt_sim)

###

# PRE-REGISTERED: ADDITIONAL ANALYSES (OLS WITH COVARIATES)

# Basic OLS model - with covariates
m_basic_cov <- lm(moderate_support ~ treated + Gender + Agecohort + State + White + Black + AmIndian_AlNative + Asian + NatHawaiian_Pacific + Hispanic_Latinx + MiddleEastern + Other + Prefer_not + Education, data = exp_data)
summary(m_basic_cov)

# OLS model + attention check (interacted with treatment) - with covariates
m_attention_cov <- lm(moderate_support ~ treated * attention + Gender + Agecohort + State + White + Black + AmIndian_AlNative + Asian + NatHawaiian_Pacific + Hispanic_Latinx + MiddleEastern + Other + Prefer_not + Education, data = exp_data)
summary(m_attention_cov)

# OLS model + partisanship (interacted with treatment) - with covariates
m_parties_cov <- lm(moderate_support ~ treated * Partisanship + Gender + Agecohort + State + White + Black + AmIndian_AlNative + Asian + NatHawaiian_Pacific + Hispanic_Latinx + MiddleEastern + Other + Prefer_not + Education, data = exp_data)
summary(m_parties_cov)

# OLS model + migration attitude (interacted with treatment) - with covariates
m_migatt_sim_cov <- lm(moderate_support ~ treated + MigAttitude_simple + treated * MigAttitude_simple + Gender + Agecohort + State + White + Black + AmIndian_AlNative + Asian + NatHawaiian_Pacific + Hispanic_Latinx + MiddleEastern + Other + Prefer_not + Education, data = exp_data)
summary(m_migatt_sim_cov)

### 

# NOT PRE-REGISTERED: LOGISTIC REGRESSION

# Basic setup with logistic regression
m_basic_logit <- glm(moderate_support ~ treated, data=exp_data, 
                     family=binomial(link="logit"))
summary(m_basic_logit)

# Preparing the results table
tidy_logit <- tidy(m_basic_logit)
tidy_logit <- tidy_logit[c("term", "estimate", "std.error", "p.value")]
panderOptions('keep.trailing.zeros', TRUE)
options(scipen = 999) # I want to avoid scientific notation in results table
tidy_logit <- tidy_logit %>%
  mutate_if(is.numeric, ~ round(., 3))
pander(tidy_logit)

###

# NOT PRE-REGISTERED: ADDITIONAL ANALYSES TO EXPLORE MECHANISM

# 1. Analysis based on response time

# Zooming in on those who were treated.
treated_only <- exp_data %>% filter(treated == 1)
untreated_only <- exp_data %>% filter(treated == 0)
# View(treated_only) # 1000 observations, correct

# Sorting by response time
treated_sorted <- treated_only %>% arrange(duration)
# View(treated_sorted)

# Split into fastest 25 and the slowest 25
fastest_250 <- treated_sorted %>% slice(1:250)  # First 25 rows (fastest)
slowest_250 <- treated_sorted %>% slice(751:1000)  # Last 25 rows (slowest)

# looking at support levels
mean(fastest_250$moderate_support) # 0.772
mean(slowest_250$moderate_support) # 0.776
mean(untreated_only$moderate_support) # 0.713

# conclusion: 
# findings based on response time are inconclusive
# does not help differentiate between mechanisms

# 2. Analysis based on response order

# Looking at mean support for the moderate proposal in both treatment groups.
# Treatment group part 1 (extreme first)
# Treatment group part 2 (moderate first)
# View(exp_data)

# Treatment group part 1 = 77.3% support for moderate proposal
# saw extreme proposal first
mean(exp_data$moderate_treat1, na.rm=TRUE)

# Treatment group part 2 = 73.7% support for moderate proposal
# saw moderate proposal first
mean(exp_data$moderate_treat2, na.rm=TRUE)

# Treatment group overall = 75.5 % support for moderate proposal
mean(exp_data$moderate_treated, na.rm=TRUE)

# Control group = 71.3% support for moderate proposal
mean(exp_data$moderate_control, na.rm=TRUE)

# Overall support for moderate proposal is 73.4% across sample
mean(exp_data$moderate_support, na.rm=TRUE)

# conclusion:
# In both treatment groups, support is higher than in the control group.
# The treatment is stronger in the group that saw the extreme version first.

###############################################################################
# End. Thanks for reading.
###############################################################################

